#!/usr/bin/env python
# -*- coding:utf-8 -*-
# @FileName  :基于已callback的数据库是否有英文链接下载英文音频.py
# @Time      :2024/5/31 
# @Author    :CL
# @email     :1037654919@qq.com
# 基于已callback的数据库是否有英文链接下载英文音频

import datetime
import json
import os
import re
import hashlib
import random
import secrets
import time
import requests
from retrying import retry

from util import mongo_manager, proxies
from multiprocessing import Pool

xuexiqiangguo_item_id  = mongo_manager('xuexiqiangguo_item_id',db='public_data')

# 如果打开代理，则使用代理，注释掉下一行
# proxies = {}

def random_base36_string():
    # 生成一个随机的大整数
    random_number = secrets.randbelow(36 ** 11)  # 调整大小以匹配JS的长度
    # 转换为基数36的字符串
    return base36encode(random_number)

def base36encode(number):
    if not isinstance(number, int):
        raise TypeError('number must be an integer')

    if number < 0:
        raise ValueError('number must be positive')

    alphabet = '0123456789abcdefghijklmnopqrstuvwxyz'
    base36 = ''

    while number:
        number, i = divmod(number, 36)
        base36 = alphabet[i] + base36

    return base36 or alphabet[0]


def sha256_hash(string):
    # 创建一个 sha256 hash 对象
    hash_obj = hashlib.sha256()

    # 提供要加密的字符串，必须先将字符串转换成字节
    hash_obj.update(string.encode('utf-8'))

    # 获取十六进制格式的摘要
    hash_digest = hash_obj.hexdigest()

    return hash_digest

def get_token(article_id,url):
    # url = "https://boot-video.xuexi.cn/voice/12341945533476085762191005/p/ee33903560c7c64fc2a0884620821d12-c0de0de695fa4740a27ac0d4835a48cc-2.mp3"
    u = 103 # 固定
    timeStamp = int(time.time() * 1000)
    c = f'{article_id}_{random_base36_string()}_{str(timeStamp)}_{str(timeStamp+random.randint(1000, 3000))}'
    # c = "5073461499409858383_h14ywy18rjk_1715407206630_1715407214178"
    f_Y1 = "1d66f6b2-442a-4834-ba8c-c265e8769931" # 固定
    ciphertext = f"{url}_{u}_{c}_{f_Y1}"
    result = sha256_hash(ciphertext)
    return c,result

@retry(stop_max_attempt_number=3, wait_fixed=3000)
#基于id 获取音频链接
def get_mp3_url(item_id = '9056979967141844798',res_url="https://boot-video.xuexi.cn/voice/12377066921295585282191005/p/a842e33fc9b3085056db156dcf3dc36c-3c26367e0dde4166913fab8bf3162b02-2.mp3"):

    headers = {
        "User-Agent": "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:125.0) Gecko/20100101 Firefox/125.0",
        "Accept": "*/*",
        "Accept-Language": "zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2",
        "Accept-Encoding": "gzip, deflate, br",
        "content-type": "application/json",
        "x-lwp-uid": "1715234954298883@pc-live",
        "Origin": "https://www.xuexi.cn",
        "Connection": "keep-alive",
        "Referer": "https://www.xuexi.cn/",
        "Sec-Fetch-Dest": "empty",
        "Sec-Fetch-Mode": "cors",
        "Sec-Fetch-Site": "same-site"
    }
    cookies = {
        "__UID__": "240797d0-0daf-11ef-801f-9b654f775548",
        "y-open-ua": "TorchApp%252F1.0%2520OSType%252FUbuntu%2520XueXi%252F2.19.0%2520Device%252Fxxqgpc%2520Ubuntu%252Fundefined(Ubuntu%253Bzh-CN)%2520lang%252Fzh-CN%2520pc.js%252F1706580237911%2520Scene%252Fpc",
        "y-open-did": "c51694c6aa91ed0bf155acef",
        "aliyungf_tc": "2864be1da263baaf8b24ea182f7e17af6f4aa8d49924331defcf8939fcd64bf0",
        "acw_tc": "ac11000117152349275265641ed796c8fe1782e8a93fa193d162d4c55568ba"
    }
    url = "https://gw-proxy-api.xuexi.cn/v1/api/exchangeAuthUrl"

    request_id,y_token = get_token(item_id,res_url)
    data = {
        "res_type": 103,
        "res_url": res_url,
        "request_id": request_id,
        "item_id": item_id,
        "y_uid": request_id,
        "y_token": y_token
    }
    data = json.dumps(data, separators=(',', ':'))
    response = requests.post(url, headers=headers, cookies=cookies, data=data,proxies=proxies)

    print(response.url,res_url,response)
    return response.json()
#基于id_url 获取callback callback返回类json，可以理解成包含所想要的一切信息
@retry(stop_max_attempt_number =3,wait_fixed=3000)
def callback(url = "https://boot-source.xuexi.cn/data/app/5073461499409858383.js"):
    headers = {
        "Accept": "*/*",
        "Accept-Language": "zh-CN,zh;q=0.9",
        "Cache-Control": "no-cache",
        "Connection": "keep-alive",
        "Pragma": "no-cache",
        "Referer": "https://www.xuexi.cn/",
        "Sec-Fetch-Dest": "script",
        "Sec-Fetch-Mode": "no-cors",
        "Sec-Fetch-Site": "same-site",
        "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
        "sec-ch-ua": "\"Chromium\";v=\"124\", \"Google Chrome\";v=\"124\", \"Not-A.Brand\";v=\"99\"",
        "sec-ch-ua-mobile": "?0",
        "sec-ch-ua-platform": "\"Linux\""
    }
    cookies = {
        "__UID__": "15344de0-11ef-11ef-816a-d722d43974a4",
        "y-open-ua": "TorchApp%252F1.0%2520OSType%252FLinux%2520XueXi%252F2.19.0%2520Device%252Fxxqgpc%2520Linux%252Fx86_64(Linux%253Bzh-CN)%2520lang%252Fzh-CN%2520pc.js%252F1706580237911%2520Scene%252Fpc",
        "y-open-did": "72da71dda21bdf517dd93842"
    }
    # url = "https://boot-source.xuexi.cn/data/app/9056979967141844798.js"
    _st = int(time.time()*1000 +random.randint(1,999))
    params = {
        "callback": "callback",
        "_st": str(_st)
    }
    # proxies= get_proxy()
    response = requests.get(url, headers=headers, cookies=cookies, params=params,proxies=proxies,timeout=10)

    print(response.url,response)
    if response.status_code == 200:
        return response.text


def  run(seed): #基于item_id下载音频
    item_id,label = seed['_id'],seed['label']
    print(seed)
    today = datetime.datetime.now().strftime('%Y%m%d')
    path = f'/media/chenglei3/77D014CED257D1411/xuexiqiangguo_en/xuexiqiangguo_{today}/{label}/'
    os.makedirs(path, exist_ok=True)

    title = seed['title'].replace('|', '_').replace('：', '_').replace('!','_').replace('"',"_").replace(':',"_")
    # 使用正则表达式将特殊符号替换为下划线
    title = re.sub(r'[\/\\\:\*\?\"\<\>\|]', '_', title)
    if len(title) > 50:
        title = title[:50]
    url = ''
    if len(seed['voices']) == 1:
        print('只有一个音频（普通话）')
        xuexiqiangguo_item_id.updateOne({"_id": item_id}, {"status_en": 'nodata'})
        return
    else:
        for voice in seed['voices']:
            if voice['voice_meta_settings']['voice_language'] == 'english':  # 英语
                url = voice['voice_storage_info'][0]['url']
                break
    if url:
        try:
            down_mp3_url = get_mp3_url(item_id=str(item_id), res_url=url)['data']['auth_url']
            response = requests.get(down_mp3_url,proxies=proxies,timeout=10)
            if response.status_code == 200:
                # 写入音频文件
                with open(f'{path}/{title}.mp3', 'wb') as file:
                    file.write(response.content)
            xuexiqiangguo_item_id.updateOne({"_id": item_id}, {"status_en": 'done','down_mp3_url_en':down_mp3_url})
        except:
            xuexiqiangguo_item_id.updateOne({"_id": item_id}, {"status_en": 'error'})
    else:
        print('无英文音频')
        xuexiqiangguo_item_id.updateOne({"_id": item_id}, {"status_en": 'nodata'})

#获取频道列表
def  get_channel(url = "https://www.xuexi.cn/lgdata/channel-list.json"):
    headers = {
        "accept": "application/json",
        "accept-language": "zh-CN,zh;q=0.9",
        "cache-control": "no-cache",
        "pragma": "no-cache",
        "priority": "u=1, i",
        "referer": "https://www.xuexi.cn/lgpage/detail/index.html?id=13624761187570256842&item_id=13624761187570256842",
        "sec-ch-ua": "\"Chromium\";v=\"124\", \"Google Chrome\";v=\"124\", \"Not-A.Brand\";v=\"99\"",
        "sec-ch-ua-mobile": "?0",
        "sec-ch-ua-platform": "\"Linux\"",
        "sec-fetch-dest": "empty",
        "sec-fetch-mode": "cors",
        "sec-fetch-site": "same-origin",
        "user-agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36"
    }
    cookies = {
        "__UID__": "76cc1450-11ba-11ef-a76a-cda34ca49ac0",
        "_bl_uid": "ROlROwwC6ba0pCb64cRgp6U1Uqh4",
        "y-open-ua": "TorchApp%252F1.0%2520OSType%252FLinux%2520XueXi%252F2.19.0%2520Device%252Fxxqgpc%2520Linux%252Fx86_64(Linux%253Bzh-CN)%2520lang%252Fzh-CN%2520pc.js%252F1706580237911%2520Scene%252Fpc",
        "y-open-did": "619b1c38b09caeb3cb8acafc"
    }

    params = {
        "_st": "28594764",
        "js_v": "1706580237911"
    }
    response = requests.get(url, headers=headers, cookies=cookies, params=params)

    print(response.url,response)
    return response.json()
#获取item_id 基于总类
def get_lgdata(url =None):

    headers = {
        "Accept": "application/json",
        "Accept-Language": "zh-CN,zh;q=0.9",
        "Cache-Control": "no-cache",
        "Connection": "keep-alive",
        "Pragma": "no-cache",
        "Referer": "https://www.xuexi.cn/xxqg.html?id=e55ff0028ab0406e948cb0be9a8cae28",
        "Sec-Fetch-Dest": "empty",
        "Sec-Fetch-Mode": "cors",
        "Sec-Fetch-Site": "same-origin",
        "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
        "sec-ch-ua": "\"Chromium\";v=\"124\", \"Google Chrome\";v=\"124\", \"Not-A.Brand\";v=\"99\"",
        "sec-ch-ua-mobile": "?0",
        "sec-ch-ua-platform": "\"Linux\""
    }
    cookies = {
        "__UID__": "31546700-10fd-11ef-aae7-c10cd7dd0de8"
    }
    if url is None:
        url = f"https://www.xuexi.cn/lgdata/e55ff0028ab0406e948cb0be9a8cae28.json?_st=28593108&js_v={int(time.time()) * 1000}"
        # params = {
        #     "_st": "28593108",
        #     "js_v": str(int(time.time()) * 1000)
        # }
    # response = requests.get(url, headers=headers, cookies=cookies, params=params)
    else:
        id_ = url.split('.json')[0].split('lgdata/')[-1]
        if '/' in id_:
            headers["Referer"] =str( 'https://www.xuexi.cn/' + id_ + '.html')
        else:
            headers["Referer"] =  f"https://www.xuexi.cn/xxqg.html?id={id_}"
    response = requests.get(url, headers=headers, cookies=cookies)

    print(response.url,response)
    return response.json()

def linshi():#临时数据库维护
    xuexiqiangguo_item_id.deleteMany({'voices': None})
    path = '/home/chenglei3/work/data/xuexiqiangguo_20240514/'
    for label in os.listdir(path):
        label_path = path + label + '/'
        print(label_path)
        for item_id in os.listdir(label_path):
            try:
                xuexiqiangguo_item_id.insertOne({"_id": item_id, 'label': label, 'item_id': item_id, 'status': 'done'})
            except Exception as e:
                print(e
                      )
                xuexiqiangguo_item_id.updateOne({"_id": item_id},
                                                {"_id": item_id, 'label': label, 'item_id': item_id, 'status': 'done'})

def downdataByitemid(num):
    print(f'这是第{num}进程')
    #临时
    num = 1
    # while True:
    while num> 0:
        seed =xuexiqiangguo_item_id.find_one_and_update({'voices': {'$exists': True},'status_en':None},{'status_en':'running'})
        print(seed)
        if seed:
            run(seed)
        else:
            break
        num -=1
def main():
    pool = Pool(processes=5)
    numss = [i for i in range(1, 6)]
    pool.map(downdataByitemid, numss)
    pool.close()
    pool.join()
# 统计含有英文链接的个数：    92602
def linshi():
    count = 0
    for item in xuexiqiangguo_item_id.findAll({'voices': {'$exists': True}}):
        print(item)
        if len(item['voices']) > 1:
            for voice in item['voices']:
                if voice['voice_meta_settings']['voice_language'] == 'english':
                    count +=1
                    break
        if count > 20:
            break
    print('统计含有英文链接的个数  ',count)
if __name__ == '__main__':
    print()
    # res= get_channel()
    # print(res)
    main()
    # linshi()
    xuexiqiangguo_item_id.close()


